import scrapy
from scrapy import Request
from allstocksname import g_StockNameDict

class DmozSpider(scrapy.Spider):
    name = "basic"
    allowed_domains = ["xueqiu.com"]
    start_urls = []
    
    headers = {
        'Connection': 'keep - alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36'
    }
    
    def __init__(self):
        for key in g_StockNameDict.keys():
            DmozSpider.start_urls.append('https://xueqiu.com/S/%s'%(key))
            
    def make_requests_from_url(self, url):
        return Request(url, headers = DmozSpider.headers)

    def parse(self, response):
        filename = response.url.split("/")[-1]
        filename = r"E:/temp/basic/" + filename
        
        with open(filename, 'wb') as f:
            f.write(response.body)
            
            
            